#-*- coding: UTF-8 -*-
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
from scipy.stats import norm
from sklearn.preprocessing import StandardScaler
from scipy import stats
import warnings
warnings.filterwarnings('ignore')

house_sale = '../data/train.csv'
data = pd.read_csv(house_sale)

#缺失数据处理
total= data.isnull().sum().sort_values(ascending=False)
percent = (data.isnull().sum()/data.isnull().count()).sort_values(ascending=False)
missing_data = pd.concat([total, percent], axis=1, keys=['Total','Percent'])
print missing_data.head(20)

data= data.drop((missing_data[missing_data['Total'] > 1]).index,1)
data= data.drop(data.loc[data['Electrical'].isnull()].index)
print data.isnull().sum().max() #justchecking that there's no missing data missing...

#异常值
#单因素分析
saleprice_scaled= StandardScaler().fit_transform(data['SalePrice'][:,np.newaxis]);
low_range = saleprice_scaled[saleprice_scaled[:,0].argsort()][:10]
high_range= saleprice_scaled[saleprice_scaled[:,0].argsort()][-10:]
print('outer range (low) of the distribution:')
print low_range
print '\nouter range (high) of thedistribution:'
print high_range

#双变量分析
#1. ‘GrLivArea’和’SalePrice’双变量分析
var = 'GrLivArea'
GrLivArea = pd.concat([data['SalePrice'], data[var]], axis=1)
GrLivArea.plot.scatter(x=var, y='SalePrice', ylim=(0,800000));
plt.show()

data.sort_values(by = 'GrLivArea',ascending = False)[:2]
data = data.drop(data[data['Id'] == 1299].index)
data = data.drop(data[data['Id'] == 524].index)

#‘TotalBsmtSF’和’SalePrice’双变量分析
var = 'TotalBsmtSF'
TotalBsmtSF = pd.concat([data['SalePrice'],data[var]], axis=1)
TotalBsmtSF.plot.scatter(x=var, y='SalePrice',ylim=(0,800000));
plt.show()

sns.distplot(data['SalePrice'], fit=norm);
plt.show()
fig = plt.figure()
res = stats.probplot(data['SalePrice'], plot=plt)
plt.show()
data['SalePrice']= np.log(data['SalePrice'])

sns.distplot(data['SalePrice'], fit=norm);
plt.show()
fig = plt.figure()
res = stats.probplot(data['SalePrice'], plot=plt)
plt.show()

#3. ‘TotalBsmtSF’绘制直方图和正态概率曲线图：
sns.distplot(data['TotalBsmtSF'],fit=norm);
plt.show()
fig = plt.figure()
res = stats.probplot(data['TotalBsmtSF'],plot=plt)
plt.show()

data['HasBsmt']= pd.Series(len(data['TotalBsmtSF']), index=data.index)
data['HasBsmt'] = 0
data.loc[data['TotalBsmtSF']>0,'HasBsmt'] = 1

#进行对数变换
data['TotalBsmtSF']= np.log(data['TotalBsmtSF'])

#绘制变换后的直方图和正态概率图
#sns.distplot(data['TotalBsmtSF'], fit=norm);
#fig = plt.figure()
#res = stats.probplot(data['TotalBsmtSF'], plot=plt)
#plt.show()

#同方差性
#1. ‘SalePrice’ 和 ‘GrLivArea’同方差性
plt.scatter(data['GrLivArea'],data['SalePrice']);
plt.show()

#2.’SalePrice’ with ‘TotalBsmtSF’同方差性
plt.scatter(data[data['TotalBsmtSF']>0]['TotalBsmtSF'], data[data['TotalBsmtSF']>0]['SalePrice']);
plt.show()

#将类别变量转换为虚拟变量：
data = pd.get_dummies(data)
print data